#!/bin/sh
#####################################################################################
# googdiff.sh - Script to (1) detect new URLs appearing in Google front page 
#               search results when compared to a previous set of search
#               results and (2) track rank order for URLs appearing on the 
#               Google front page results.  This is usefule for policing 
#               ones Internet identity and for determining how ones pages
#               change in rank over the course of time.  
#
# Author  - Lamar Spells (lamar.spells@gmail.com) 
# Blog    - http://foxtrot7security.blogspot.com
# Twitter - lspells
#
# Copyright (c) 2012, Lamar Spells
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# - Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
# - Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.
#
##################################################################################### 
# Modifications - 
# 22 March 2013 - Modified to support changes to Google search output HTML
#####################################################################################

USAGE="USAGE: googdiff.sh [-d <dir> [-s] -p -t <terms>] | [-o <filename> -n <filename>]\n
 Command line arguments \n
 ---------------------- \n
 Compare current to previous results (or create a baseline for use later):\n
 -d <directory> - results directory\n
 -s - save current results in working directory (optional)\n
 -p - compare present results to most recent previous result;\n
\tif no previous results, create a baseline using current results\n
 -t <search terms> - search term string\n\n
 ---OR---\n\n
 Compare two specific files: \n
 -o <filename> - old results file\n
 -n <filename> - new results file\n\n"

# Figure out where we live.
INSTALL_DIR=`dirname $0`
if [ $INSTALL_DIR = "." ] ; then
   INSTALL_DIR=`pwd`
fi
# Make sure our supporting components
# are available and ready to go...
SEARCH_GOOGLE=${INSTALL_DIR}/search_google.pl
if [ ! -x $SEARCH_GOOGLE ] ; then
  echo "Cannot find search utility $SEARCH_GOOGLE or utility not executable"
  exit 1
fi

# Make sure minimum command line args are present
if [ $# -lt 4 ] ; then
   echo $USAGE
   exit 1
fi

# Parse the command line according to USAGE supplied...
parg=0
sarg=0
dargset=0
oargset=0
nargset=0
targset=0
while getopts d:spo:n:t: f
do
   case $f in
         d)  dargset=1
             darg=$OPTARG ;;
         s)  sarg=1 ;;
         p)  parg=1 ;;
         o)  oargset=1
             oarg=$OPTARG ;;
         n)  nargset=1
             narg=$OPTARG ;;
         t)  targset=1
             targ=$OPTARG ;;
        \?)  echo $USAGE; exit 1;;
   esac
done
shift `expr $OPTIND - 1`

if [ $dargset -eq 0 ] ; then
   # No directory specified -- we are comparing old files
   # cant specify -p ; must specify -o and -n
   op_mode=0
   if [ $oargset -ne 1 ] ; then
      echo "-o <file> is required if -d <dir> is not supplied"
      exit 1
   fi
   if [ $nargset -ne 1 ] ; then
      echo "-n <file> is required if -d <dir> is not supplied"
      exit 1
   fi
   if [ ! -r $oarg ] ; then
      echo "File $oarg is not readable!"
      exit 1
   fi
   if [ ! -r $narg ] ; then
      echo "File $narg is not readable!"
      exit 1
   fi
else
   # We are going to be retrieving results... 
   # directory better be present with old results 
   # and to store the new results
   op_mode=1
   if [ ! -d $darg ] ; then
      echo "Directory $darg not found"
      exit 1
   fi
 
   # Make sure you can read / write / execute on the directory
   if [ ! -r $darg -o ! -w $darg -o ! -x $darg ] ; then
      echo "Check for read/write/exec permission on $darg"
      exit 1
   fi

   if [ $parg -eq 0 ] ; then
      echo "If you specify -d <dir> then -p is required"
      exit 1
   fi
   # -t search terms are only required if 
   # you are searching "live" and in op_mode 1
   if [ $targset -eq 0 ] ; then
      echo "Search terms (-t <terms>) are required if -d <dir> is used"
      exit 1
   fi
   # -s for save current results is optional at all times --
   # no need to check for that at all.
fi

# Now we know what we are doing... 
# account for the different modes 
# of operation by setting up the  
# input files for the later comparison
if [ $op_mode -eq 1 ] ; then
   # Determine the previous results to use --
   # If no previous results exist, simply make a 
   # baseline and perform no further processing.
   PREVIOUS_RESULTS=`ls -ltr $darg/* 2> /dev/null | tail -1 | awk '{print $NF}'`
   if [ "$PREVIOUS_RESULTS" = "" ] ; then
      echo "No previous results found - creating baseline"
      BASELINE_ONLY=1
      sarg=1
   else
      BASELINE_ONLY=0
   fi

   # Create the temp file for current results
   CURRENT_RESULTS=${TMPDIR:-"/tmp"}/${targ}.`date +%Y-%B-%d`

   # Retrieve the current results  - omitting all the google and youtube links 
   # automatically embedded in the front page results. 
   $SEARCH_GOOGLE $targ | grep '^http://www.google.com/url' | sed -e 's!http://www.google.com/url?q=!!g' |awk -F'&sa=' '{print $1}' | grep -v googleusercontent.com | grep -v youtube.com | grep -v '/settings/' > $CURRENT_RESULTS

   # Save the current results in the directory specified by -d if -s is given
   if [ $sarg -eq 1 ] ; then
      cp $CURRENT_RESULTS $darg
   fi

else
   BASELINE_ONLY=0
   PREVIOUS_RESULTS=$oarg
   CURRENT_RESULTS=$narg
fi

if [ $BASELINE_ONLY -eq 1 ] ; then
   echo "Baseline results stored in $darg/`basename $CURRENT_RESULTS`"
   exit 0
fi

echo "Google Search Page Rank / New URL Report"
echo "========================================"
echo "Executed On   : `date +%Y-%B-%d`"
echo "Search Term(s): ${targ:-"--Not Provided--"}"
echo "Previous Results: `basename $PREVIOUS_RESULTS`"
echo "Current Results : `basename $CURRENT_RESULTS`\n"

# Actually compare the results... Finally. 
cat $CURRENT_RESULTS | awk -v old_input="$PREVIOUS_RESULTS" \
' 
BEGIN { 
         line_num=0;

         # Create two sets of arrays for 
         # convenience.  newurls & oldurls
         # are associative arrays that use the 
         # URL as the index.  This is handy for
         # quickly determining whether or not a 
         # given URL in the current search results
         # was in the list of URLs previously 
         # seen.  The "inorder" arrays are used 
         # to track previous and current search 
         # rankings for the URLs returned for 
         # search terms.  This lets you determine
         # changes to rank order from week to week.
         newurls[""]=0;
         newinorder[""]=0;
         oldurls[""]=0;
         oldinorder[""]=0;

         # Problem indicators
         problemurls[""]=0;
      }
      { 
         # New or current google search results
         # arrive via STDIN from the perl program.
         newurls[$0]=++line_num;
         newinorder[line_num]=$0
      }
END   {
         # We have to go get the previous search
         # results from a file so we do that after
         # we have loaded up the current results.
         line_num=0;
         while ( (getline oldurl < old_input ) > 0 )
         {
            oldurls[oldurl]=++line_num;
            oldinorder[line_num]=oldurl;
         }
         new_urls_found=0
         for (i=1; i < length(newinorder); i++) 
         {
            # If the previous page rank is 0, we know this is a brand new
            # new URL that we are seeing for the first time.  This might 
            # be a problem, especially if the rank is high.  It could be a 
            # link introduced by a "Black Hat" SEO hacker looking to exploit
            # our identity... any new URL that shows up in the top 10 warrants
            # a careful manual review.  
            if ( oldurls[newinorder[i]]==0 ) new_urls_found++;
         }
         if ( new_urls_found > 0 )
         {
            printf("\nWARNING: %d new URL(s) found in Google search results.\n",new_urls_found);
            printf("         New URLs should be reviewed manually to ensure\n");
            printf("         that your online identity remains secure.\n\n");
            printf("URL(s) Requiring Manual Review\n");
            printf("==============================\n");
            for (i=1; i < length(newinorder); i++) 
            {
               if ( oldurls[newinorder[i]]==0 ) 
                  printf("- %s\n", newinorder[i]);
            }
         }
         else
         {
            printf("\nNo new URLs found in Google search results.\n");
         }
         printf("\nRank Changes From Previous Results\n");
         printf("==================================\n\n");
         printf("Current Rank  Previous Rank  URL\n");
         printf("============  =============  ===================\n");
         for (i=1; i < length(newinorder); i++) 
         {
            printf("%0.2d            %0.2d             %s\n", i, oldurls[newinorder[i]], newinorder[i]);
         }
         printf("\nCurrent Google Search Results\n");
         printf("=============================\n");
         for (i=1; i < length(newinorder); i++) printf("%d \t %s \n", i, newinorder[i]);

         printf("\nPrevious Google Search Results\n");
         printf("==============================\n");
         for (i=1; i < length(oldinorder); i++) printf("%d \t %s \n", i, oldinorder[i]);
      } ' 

# Do not erase current results if just comparing two files...
# otherwise, ditch the temporary file.
if [ $op_mode -eq 1 ] ; then
   rm -f $CURRENT_RESULTS > /dev/null 2>&1 
fi

exit 0;
